package com.godtree.service.impl;

import static cn.hutool.core.text.StrPool.EMPTY_JSON;
import static com.godtree.constant.Constants.DROP_LIST;
import static com.godtree.constant.Constants.ONE_HUNDRED;
import static com.godtree.constant.Constants.ONE_STR;
import static com.godtree.constant.Constants.PATTERN_REGEX_PERCENT;
import static com.godtree.constant.Constants.PATTERN_REGEX_TIME;
import static com.godtree.constant.Constants.ZERO_STR;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.convert.Convert;
import cn.hutool.core.exceptions.ExceptionUtil;
import cn.hutool.core.lang.Tuple;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.thread.ThreadUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.ReflectUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.extra.pinyin.PinyinUtil;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONConfig;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.godtree.dao.RankCrawlerCollectionMapper;
import com.godtree.dto.RankCrawlerCollectionDTO;
import com.godtree.dto.RankCrawlerCollectionDTO.ScoreDTO;
import com.godtree.dto.RankDataNameDTO;
import com.godtree.exception.RankAliasRepeatException;
import com.godtree.exception.RankException;
import com.godtree.model.RankCompare;
import com.godtree.model.RankCrawlerCollection;
import com.godtree.model.RankData;
import com.godtree.model.RankData.Fields;
import com.godtree.model.RankDataMapping;
import com.godtree.model.RankDrop;
import com.godtree.model.RankItem;
import com.godtree.model.RankScore;
import com.godtree.model.Ranks;
import com.godtree.service.IRankCompareService;
import com.godtree.service.IRankCrawlerCollectionService;
import com.godtree.service.IRankDataMappingService;
import com.godtree.service.IRankDataService;
import com.godtree.service.IRankDropService;
import com.godtree.service.IRankItemService;
import com.godtree.service.IRankScoreService;
import com.godtree.service.IRanksService;
import com.godtree.util.UnitConvertUtil;
import com.google.common.base.CharMatcher;
import com.google.common.collect.HashBiMap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.xxl.job.core.context.XxlJobHelper;
import com.xxl.job.core.util.GsonTool;
import java.lang.reflect.Field;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import lombok.RequiredArgsConstructor;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

/**
 * <p>
 * 服务实现类
 * </p>
 *
 * @author lisicheng
 * @since 2022-07-19
 */
@Service
@RequiredArgsConstructor
public class RankCrawlerCollectionServiceImpl extends
    ServiceImpl<RankCrawlerCollectionMapper, RankCrawlerCollection> implements
    IRankCrawlerCollectionService {

  private final IRanksService ranksService;
  private final IRankItemService rankItemService;
  private final IRankDataService rankDataService;
  private final IRankDropService rankDropService;
  private final IRankScoreService rankScoreService;
  private final IRankCompareService rankCompareService;
  private final IRankDataMappingService rankDataMappingService;
  private final MongoTemplate mongoTemplate;

  @Override
  @Transactional(rollbackFor = Exception.class)
  public void rankDataByMysql(String rankId) {

    // 如果rankId为空，直接返回
    if (StrUtil.isBlank(rankId)) {
      XxlJobHelper.handleFail("rankId为空");
      return;
    }

    // 根据rankId（去掉前后空格）获取排名
    Ranks ranks = ranksService.getById(StrUtil.trim(rankId));
    // 如果根据rankId未获取到排名，直接返回
    if (ranks == null) {
      XxlJobHelper.handleFail(StrUtil.format("根据rankId未获取到排名：{}", rankId));
      return;
    }

    // 查询排名列
    List<RankItem> rankItemList = rankItemService.list(Wrappers.<RankItem>lambdaQuery()
        .select(RankItem::getId, RankItem::getName, RankItem::getOrderBy, RankItem::getUnitConvert,
            RankItem::getClearUpdate).eq(RankItem::getRanks, rankId)
        .eq(RankItem::getDelFlag, ZERO_STR).orderByAsc(RankItem::getSort));

    String name = ranks.getName();
    XxlJobHelper.log("排名【{}|{}】排名列【{}】条", name, rankId, rankItemList.size());

    List<RankData> rankDataList = Lists.newArrayList();
    while (true) {
      List<RankData> list = constructRankDataWithMysql(rankId, rankItemList);
      if (list == null) {
        break;
      }
      rankDataList.addAll(list);
    }

    if (rankDataList.size() == 0) {
      XxlJobHelper.log("排名【{}|{}】未更新数据", name, rankId);
      return;
    }

    // 排名映射
    Map<String, String> rankDataMappingMap = rankDataMappingService.list(
            Wrappers.<RankDataMapping>lambdaQuery().eq(RankDataMapping::getRankId, rankId)
                .select(RankDataMapping::getName, RankDataMapping::getAlias)).stream()
        .collect(Collectors.toMap(RankDataMapping::getName, RankDataMapping::getAlias));
    // 去重，如果存在重复，并且同列都有值，取第一个
    rankDataList = duplicateRemoval(rankDataList, rankDataMappingMap);

    XxlJobHelper.log("排名【{}|{}】计算数据{}条", name, rankId, rankDataList.size());
    save(rankId, name, rankDataList, rankDataMappingMap, rankItemList);
    // TODO: 2022/7/19 文章更新推送消息
    // 参考：com.jeeplus.modules.rank.data.service.RankDataService#saves
    ranksService.updateKeyWords(rankId);
    // 更新排名
    for (RankItem rankItem : rankItemList) {
      rankScoreService.updateRankValue(rankId, rankItem.getId(), "1".equals(rankItem.getOrderBy()));
    }
    // 更新排名默认分数
    rankDataService.updateDefaultRankScore(rankId);

  }

  @Override
  public void rankDataByMongo(String rankId) {

    if (StrUtil.isBlank(rankId)) {
      XxlJobHelper.handleFail("rankId为空");
      return;
    }

    Ranks ranks = ranksService.getById(StrUtil.trim(rankId));
    if (ranks == null) {
      XxlJobHelper.handleFail(StrUtil.format("根据rankId未获取到排名：{}", rankId));
      return;
    }

    long count = mongoTemplate.count(new Query(Criteria.where("rank_id").is(rankId)), "rank");
    if (count == 0) {
      XxlJobHelper.log("排名【{}|{}】未更新数据", ranks.getName(), rankId);
      return;
    }

    // 查询排名列
    List<RankItem> rankItemList = rankItemService.list(Wrappers.<RankItem>lambdaQuery()
        .select(RankItem::getId, RankItem::getName, RankItem::getOrderBy, RankItem::getUnitConvert,
            RankItem::getClearUpdate).eq(RankItem::getRanks, rankId)
        .eq(RankItem::getDelFlag, ZERO_STR).orderByAsc(RankItem::getSort));

    String name = ranks.getName();
    XxlJobHelper.log("排名【{}|{}】排名列【{}】条", name, rankId, rankItemList.size());

    List<RankData> rankDataList = Lists.newArrayList();
    int index = 0;
    int limit = 100;
    while (true) {
      List<RankData> consume = constructRankDataWithMongoDB(rankId, rankItemList,
          mongoTemplate.find(
              new Query(Criteria.where("rank_id").is(rankId)).skip(index).limit(limit), Map.class,
              "rank"));
      if (consume == null) {
        break;
      }
      index += limit;
      rankDataList.addAll(consume);
    }

    // 删除数据前，再次判断是否有数据更新
    if (count != mongoTemplate.count(new Query(Criteria.where("rank_id").is(rankId)), "rank")) {
      XxlJobHelper.log("排名【{}|{}】数据消费数据时有数据更新，停止消费", name, rankId);
      return;
    }
    // 消费完数据后，删除
    mongoTemplate.remove(new Query(Criteria.where("rank_id").is(rankId)), "rank");

    if (rankDataList.size() == 0) {
      XxlJobHelper.log("排名【{}|{}】未更新数据", name, rankId);
      return;
    }

    // 排名映射
    Map<String, String> rankDataMppingMap = rankDataMappingService.list(
            Wrappers.<RankDataMapping>lambdaQuery().eq(RankDataMapping::getRankId, rankId)
                .select(RankDataMapping::getName, RankDataMapping::getAlias)).stream()
        .collect(Collectors.toMap(RankDataMapping::getName, RankDataMapping::getAlias));
    // 去重，如果存在重复，并且同列都有值，取第一个
    rankDataList = duplicateRemoval(rankDataList, rankDataMppingMap);

    XxlJobHelper.log("排名【{}|{}】计算数据{}条", name, rankId, rankDataList.size());
    // 保存数据
    this.save(rankId, name, rankDataList, rankDataMppingMap, rankItemList);
    // TODO: 2022/7/19 文章更新推送消息
    // 参考：com.jeeplus.modules.rank.data.service.RankDataService#saves
    ranksService.updateKeyWords(rankId);
    // 更新排名
    for (RankItem rankItem : rankItemList) {
      rankScoreService.updateRankValue(rankId, rankItem.getId(), "1".equals(rankItem.getOrderBy()));
    }
    // 更新排名默认分数
    rankDataService.updateDefaultRankScore(rankId);
  }

  /**
   * 构造排名数据（通过Mysql表）
   *
   * @param rankId       排名ID
   * @param rankItemList 排名列
   * @return 排名数据
   */
  @Transactional(rollbackFor = Exception.class)
  public List<RankData> constructRankDataWithMysql(String rankId, List<RankItem> rankItemList) {

    // 抓取的数据
    List<RankCrawlerCollectionDTO> rankCrawlerList = buildRankCrawlerList(rankItemList,
        () -> baseMapper.getData(rankItemList, rankId));

    // 如果没有数据，直接返回
    if (CollUtil.isEmpty(rankCrawlerList)) {
      return null;
    }

    // 默认排名列
    String defaultRankItemId = rankItemList.stream().findFirst().map(RankItem::getId).orElse("");

    // 通过爬虫数据构造排名数据
    List<RankData> rankDataList = rankCrawlerList.stream()
        // 过滤掉空名称数据
        .filter(item -> StrUtil.isNotBlank(item.getName())).map(item -> {
          RankData rankData = buildRankData(item, rankId, defaultRankItemId);
          // 顶部筛选项
          buildRankDropList(rankData, rankId);
          // 排名列
          buildRankScoreList(rankData, item, rankItemList);
          return rankData;
        }).collect(Collectors.toList());

    // 随机休眠，避免任务在执行中正在导入数据，导致执行任务的数据不完整
    ThreadUtil.sleep(TimeUnit.SECONDS.toMillis(RandomUtil.randomInt(2, 3)));

    // 消费完数据后，删除
    removeByIds(
        rankCrawlerList.stream().map(RankCrawlerCollectionDTO::getId).collect(Collectors.toList()));

    return rankDataList;
  }

  /**
   * 构造排名数据（通过MongoDB表）
   *
   * @param rankId       排名ID
   * @param rankItemList 排名列
   * @param mongoData    mongo数据
   * @return 排名数据
   */
  @SuppressWarnings("all")
  public List<RankData> constructRankDataWithMongoDB(String rankId, List<RankItem> rankItemList,
      List<Map> mongoData) {
    // 抓取的数据
    List<RankCrawlerCollectionDTO> rankCrawlerList = buildRankCrawlerList(rankItemList,
        () -> mongoData);

    if (CollUtil.isEmpty(rankCrawlerList)) {
      return null;
    }

    // 默认排名列
    String defaultRankItemId = rankItemList.stream().findFirst().map(RankItem::getId).orElse("");

    return rankCrawlerList.stream().filter(item -> StrUtil.isNotBlank(item.getName())).map(item -> {
      RankData rankData = buildRankData(item, rankId, defaultRankItemId);
      // 顶部筛选项
      buildRankDropList(rankData, rankId);
      // 排名列
      buildRankScoreList(rankData, item, rankItemList);
      return rankData;
    }).collect(Collectors.toList());
  }

  /**
   * 排名数据入库
   *
   * @param rankId             排名ID
   * @param rankName           排名名称
   * @param rankDataList       排名列数据
   * @param rankDataMappingMap 排名列数据映射 Map<name, alias>
   */
  @Transactional(rollbackFor = Exception.class)
  public void save(String rankId, String rankName, List<RankData> rankDataList,
      Map<String, String> rankDataMappingMap, List<RankItem> rankItemList) {

    // db中排名下所有的排名数据
    List<RankData> dbRankDataList = getDbRankDataList(rankId, rankDataList, rankDataMappingMap);

    // 通过name映射关系，构造映射关系的Map <name, id>
    Map<String, String> rankDataDbMap = dbRankDataList.stream()
        .collect(Collectors.toMap(RankData::getName, RankData::getId));
    // 通过alias映射关系，构造映射关系的Map <alias, id>
    Map<String, String> rankDataAliasMap = dbRankDataList.stream()
        .filter(item -> ObjectUtil.isNotNull(item.getAlias()))
        .collect(Collectors.toMap(RankData::getAlias, RankData::getId));
    // 通过id映射关系，构造映射关系的Map <id, RankData>
    Map<String, RankData> rankDataDbEntityMap = dbRankDataList.stream()
        .collect(Collectors.toMap(RankData::getId, Function.identity()));
    // 通过映射关系，构造映射关系的BiMap（反转map） <alias, name>
    Map<String, List<String>> rankDataMappingBiMap = rankDataMappingMap.entrySet().stream().collect(
        Collectors.groupingBy(Entry::getValue,
            Collectors.mapping(Entry::getKey, Collectors.toList())));

    XxlJobHelper.log("排名【{}|{}】db中数据【{}】条", rankName, rankId, rankDataDbMap.size());

    // 通过匹配关系，设置id
    matchRankDataId(rankDataList, rankDataDbMap, rankDataAliasMap, rankDataMappingBiMap);

    // 与计算后的rankDataList匹配id，配对到id的，表示db中已存在，未配对到id的，表示未存在
    List<String> rankDataIdList = rankDataList.stream().map(RankData::getId)
        .filter(Objects::nonNull).collect(Collectors.toList());

    // gc
    CollUtil.clear(dbRankDataList);
    MapUtil.clear(rankDataDbMap, rankDataAliasMap, rankDataMappingBiMap);

    LocalDateTime now = LocalDateTime.now();
    // 计算排名数据与db数据融合
    List<RankData> updList = rankDataList.stream().filter(item -> item.getId() != null)
        .peek(item -> merge(rankDataDbEntityMap, now, item)).collect(Collectors.toList());
    // 设置id和创建时间
    List<RankData> addList = rankDataList.stream().filter(item -> item.getId() == null)
        .peek(item -> {
          item.setId(IdUtil.getSnowflakeNextIdStr());
          item.setCreateDate(now);
          item.setUpdateDate(now);
          item.setAlias(StrUtil.blankToDefault(item.getAlias(), null));
        }).collect(Collectors.toList());

    // 检查别名是否重复
    checkAliasRepeat(rankId, addList, updList);

    // gc
    MapUtil.clear(rankDataDbEntityMap);

    XxlJobHelper.log("排名【{}|{}】修改【{}】条，新增【{}】条", rankName, rankId, rankDataIdList.size(),
        rankDataList.size() - rankDataIdList.size());

    // 开启了清空更新的排名列
    List<String> clearUpdateIdList = rankItemList.stream()
        .filter(item -> ObjectUtil.equal(1, item.getClearUpdate())).map(RankItem::getId)
        .collect(Collectors.toList());

    // 清空更新的排名列数据
    if (CollUtil.isNotEmpty(clearUpdateIdList)) {
      // 开启了清空更新的排名列与本次更新的排名列取交集，得到最终需要清空的排名列
      Set<String> ultimatelyClearIdList = CollUtil.intersectionDistinct(clearUpdateIdList,
          rankDataList.stream().map(RankData::getRankScoreList).flatMap(Collection::stream)
              .filter(RankScore::getExist).map(RankScore::getRankItem).distinct()
              .collect(Collectors.toList()));
      if (CollUtil.isNotEmpty(ultimatelyClearIdList)) {
        rankScoreService.remove(
            Wrappers.<RankScore>lambdaQuery().in(RankScore::getRankItem, ultimatelyClearIdList));
      }
      // gc
      CollUtil.clear(clearUpdateIdList);
    }

    int batchSize = 1000;
    // 删除排名数据的下拉筛选项、排名列、排名列比较数据（附属表数据）
    if (CollUtil.isNotEmpty(rankDataIdList)) {
      rankDropService.remove(
          Wrappers.<RankDrop>lambdaQuery().in(RankDrop::getRankData, rankDataIdList));
      rankCompareService.remove(
          Wrappers.<RankCompare>lambdaQuery().in(RankCompare::getRankData, rankDataIdList));
      // 删除本次更新的排名列数据
      List<Map<String, String>> doubleInList = updList.stream().map(
              item -> item.getRankScoreList().stream().map(
                  rankScore -> ImmutableMap.of("rankItem", rankScore.getRankItem(), "rankData",
                      item.getId())).collect(Collectors.toList())).flatMap(Collection::stream)
          .collect(Collectors.toList());
      if (CollUtil.isNotEmpty(doubleInList)) {
        Lists.partition(doubleInList, batchSize)
            .forEach(item -> rankScoreService.deleteByDoubleIn(rankId, item));
      }
    }

    // 批量保存排名数据
    Lists.partition(addList, batchSize).forEach(rankDataService::saveBatchSomeColumn);
    Lists.partition(updList, batchSize).forEach(rankDataService::batchSaveOrUpdate);

    // 将排名数据id设置到排名列和顶部筛选项中
    for (RankData rankData : rankDataList) {
      String rankDataId = rankData.getId();
      for (RankDrop rankDrop : rankData.getRankDropList()) {
        rankDrop.setId(IdUtil.getSnowflakeNextIdStr());
        rankDrop.setRankData(rankDataId);
      }
      for (RankScore rankScore : rankData.getRankScoreList()) {
        rankScore.setId(IdUtil.getSnowflakeNextIdStr());
        rankScore.setRankData(rankDataId);
      }
    }

    // 批量保存顶部筛选项
    List<RankDrop> rankDropList = rankDataList.stream()
        .flatMap(item -> item.getRankDropList().stream()).collect(Collectors.toList());
    Lists.partition(rankDropList, batchSize).forEach(rankDropService::saveBatchSomeColumn);

    // 批量保存排名列
    List<RankScore> rankScoreList = rankDataList.stream()
        .flatMap(item -> item.getRankScoreList().stream()).collect(Collectors.toList());
    Lists.partition(rankScoreList, batchSize).forEach(rankScoreService::saveBatchSomeColumn);

  }

  private String buildKey(Map.Entry<RankDataNameDTO, List<RankData>> item,
      Map<String, String> mappingMap) {
    List<RankData> value = item.getValue();
    String rankId = value.get(0).getRanks();
    List<RankDataNameDTO> aliasList = value.stream().map(RankData::getAlias)
        .filter(StrUtil::isNotBlank).map(RankDataNameDTO::new).distinct()
        .collect(Collectors.toList());

    String result;
    if (value.size() == 1 || CollUtil.isEmpty(aliasList)) {
      result = value.get(0).getName();
    } else {
      List<String> nameList = value.stream().map(RankData::getName).map(RankDataNameDTO::new)
          .filter(name -> !aliasList.contains(name)).distinct().map(RankDataNameDTO::getBaseName)
          .collect(Collectors.toList());
      if (CollUtil.isEmpty(nameList)) {
        String content = StrUtil.format(
            "排名【{}】数据清洗时名称与别名映射存在错误，原数据名称集合为空，请关注！数据：\n{}", rankId,
            JSONUtil.toJsonStr(value));
        // WranMsgUtils.send(WranEnum.RANK.getTitle(), content);
        throw new RankException(content);
      }

      if (nameList.size() != 1) {
        String content = StrUtil.format(
            "排名【{}】数据清洗时名称与别名映射存在错误，原数据名称集合不为1，请关注！数据：\n{}", rankId,
            JSONUtil.toJsonStr(value));
        // WranMsgUtils.send(WranEnum.RANK.getTitle(), content);
        throw new RankException(content);
      }

      result = nameList.get(0);
    }

    return mappingMap.getOrDefault(result, result);
  }

  private String buildTags(List<String> tagList) {

    // 构建标签
    // 如果标签为空，则使用默认标签
    if (CollUtil.isEmpty(tagList)) {
      return "{}";
    }
    // 标签转换为LinkedHashMap
    LinkedHashMap<String, Object> tagMap = new LinkedHashMap<>();
    // 标签格式：标签1-值1;标签2-值2;标签3-值3...标签n-值n
    // 通过;分割，然后通过-分割，然后构建标签
    // 最后转换为JSON字符串，格式：{"标签1":"值1","标签2":"值2","标签3":"值3"...,"标签n":"值n"}
    for (int i = 0; i < tagList.size(); i++) {
      String keyValue = tagList.get(i);
      if (StrUtil.isBlank(StrUtil.trim(keyValue))) {
        tagMap.put("其他" + (i + 1), "数据生成中");
        continue;
      }
      keyValue = StrUtil.endWith(keyValue, ";") ? StrUtil.subBefore(keyValue, ";", true) : keyValue;
      String[] split = keyValue.split("-");
      if (split.length > 1) {
        String key = StrUtil.trim(split[0]);
        String value = StrUtil.trim(split[1]);
        tagMap.put(key, value);
      }
    }
    return GsonTool.toJson(tagMap);
  }

  /**
   * 通过排名列，排名爬虫数据函数构建排名爬虫数据集合
   *
   * @param rankItemList 排名列
   * @param dataSupplier 排名爬虫数据函数
   * @return {@link List<RankCrawlerCollectionDTO>}
   */
  @SuppressWarnings("all")
  private List<RankCrawlerCollectionDTO> buildRankCrawlerList(List<RankItem> rankItemList,
      Supplier<List<Map>> dataSupplier) {

    List<Map> list = dataSupplier.get();
    int rankItemListSize = rankItemList.size();

    // 构建排名爬虫数据
    // 去重实体类重写了equals和hashCode方法，忽略了id和groupDate属性
    return list.stream().map(item -> buildRankCrawler(item, rankItemListSize)).distinct()
        .collect(Collectors.toList());
  }

  /**
   * 构建排名爬虫数据
   *
   * @param item             数据(Map 动态排名列)
   * @param rankItemListSize 排名列数量
   * @return {@link RankCrawlerCollectionDTO}
   */
  private RankCrawlerCollectionDTO buildRankCrawler(@SuppressWarnings("all") Map item,
      int rankItemListSize) {

    // 排名名称模板
    String itemName = "score_{}";
    // 排名链接模板
    String itemUrl = "score_url_{}";

    @SuppressWarnings("unchecked") Map<String, Object> itemMap = item;

    // 构建标签。
    // tags_1:标签1,tags_2:标签2,tags_3:标签3...tags_n:标签n
    // 通过tags_1的key，获取最大的tags_n的key，然后通过最大的key，构建标签
    List<String> tagList = Lists.newArrayList();
    Integer tagsMax = itemMap.keySet().stream().filter(k -> StrUtil.startWith(k, "tags_"))
        .map(k -> StrUtil.replace(k, "tags_", "")).map(Convert::toInt).max(Integer::compareTo)
        .orElse(null);
    if (tagsMax != null) {
      for (int i = 0; i < tagsMax; i++) {
        tagList.add(Convert.toStr(itemMap.get(StrUtil.format("tags_{}", i + 1))));
      }
    } else {
      tagList.addAll(
          StrUtil.split(Convert.toStr(itemMap.get("tags")), ";").stream().map(str -> str + ";")
              .collect(Collectors.toList()));
    }

    // 通过排名列数量，构建排名数据（从1开始）
    List<ScoreDTO> scoreList = IntStream.rangeClosed(1, rankItemListSize).mapToObj(
            index -> new ScoreDTO(itemMap.containsKey(StrUtil.format(itemName, index)),
                Convert.toStr(itemMap.get(StrUtil.format(itemName, index))),
                Convert.toStr(itemMap.get(StrUtil.format(itemUrl, index)))))
        .collect(Collectors.toList());

    return RankCrawlerCollectionDTO.builder().id(Convert.toStr(itemMap.get("id")))
        .rankId(Convert.toStr(itemMap.get("rank_id")))
        .name(StrUtil.trim(Convert.toStr(itemMap.get("name"))))
        .alias(StrUtil.trim(Convert.toStr(itemMap.get("alias"))))
        .imageUrl(Convert.toStr(itemMap.get("image_url")))
        .drop1(Convert.toStr(itemMap.get("drop_1"))).drop2(Convert.toStr(itemMap.get("drop_2")))
        .drop3(Convert.toStr(itemMap.get("drop_3"))).tagList(tagList)
        .recommend(Convert.toStr(itemMap.get("recommend"))).scoreList(scoreList)
        .groupDate(Convert.toLocalDateTime(itemMap.get("create_time"), LocalDateTime.now()))
        .build();
  }

  /**
   * 通过排名爬虫数据构建排名数据
   *
   * @param dto               排名爬虫数据
   * @param rankId            排名ID
   * @param defaultRankItemId 默认排名列ID
   * @return {@link RankData}
   */
  private RankData buildRankData(RankCrawlerCollectionDTO dto, String rankId,
      String defaultRankItemId) {

    // 构建筛选项（最多三个）
    // 如果筛选项为空，则使用默认值。默认值：['','','']
    String drops = JSONUtil.toJsonStr(
        Lists.newArrayList(StrUtil.nullToEmpty(dto.getDrop1()), StrUtil.nullToEmpty(dto.getDrop2()),
            StrUtil.nullToEmpty(dto.getDrop3())));

    String name = dto.getName();
    // 构建排名数据
    return RankData.builder().ranks(rankId).name(name).alias(dto.getAlias())
        // 全拼音
        .fullPinyinName(PinyinUtil.getPinyin(name, ""))
        // 简拼（首字母）
        .simplePinyinName(PinyinUtil.getFirstLetter(name, "")).image(dto.getImageUrl()).drops(drops)
        .tagList(dto.getTagList()).recommendation(dto.getRecommend()).groupDate(dto.getGroupDate())
        .delFlag(ZERO_STR).showHide(ONE_STR).defaultRankItemId(defaultRankItemId)
        .rankDropList(Lists.newArrayList()).rankScoreList(Lists.newArrayList()).build();
  }

  /**
   * 构建排名筛选项
   *
   * @param rankData 排名数据
   * @param rankId   排名ID
   */
  private void buildRankDropList(RankData rankData, String rankId) {
    // 排名筛选项排序
    int sort = 0;
    // 遍历筛选项
    for (String drop : JSONUtil.toList(StrUtil.blankToDefault(rankData.getDrops(), DROP_LIST),
        String.class)) {
      // 构建排名筛选项，每次循环，排序+1（排名从0开始）
      RankDrop rankDrop = RankDrop.builder().ranks(rankId).sort(sort++).name(drop).delFlag(ZERO_STR)
          .build();
      // 添加到排名筛选项列表
      rankData.getRankDropList().add(rankDrop);
    }
  }

  /**
   * 构建排名列
   *
   * @param rankData     排名数据
   * @param dto          排名爬虫数据
   * @param rankItemList 排名列列表
   */
  private void buildRankScoreList(RankData rankData, RankCrawlerCollectionDTO dto,
      List<RankItem> rankItemList) {

    // 爬虫数据的排名列集合
    List<RankCrawlerCollectionDTO.ScoreDTO> scoreList = dto.getScoreList();

    // 遍历排名列集合
    int rankItemSize = rankItemList.size();
    int scoreSize = scoreList.size();
    for (int i = 0; i < rankItemSize; i++) {

      // 通过索引获取排名列，和爬虫数据的排名列
      RankItem rankItem = rankItemList.get(i);
      RankCrawlerCollectionDTO.ScoreDTO scoreDTO = i < scoreSize ? scoreList.get(i) : null;
      if (scoreDTO == null) {
        continue;
      }

      // 排名列排序（从1开始）
      int index = i + 1;
      // 是否是第一个排名列
      boolean first = index == 1;

      // 排名列构建器
      RankScore.RankScoreBuilder rankScoreBuilder = RankScore.builder().rankItem(rankItem.getId())
          .ranks(rankData.getRanks()).url(StrUtil.nullToEmpty(scoreDTO.getUrl())).sort(index)
          .delFlag(ZERO_STR).exist(
              scoreDTO.getExist() && (StrUtil.isNotBlank(scoreDTO.getName()) || StrUtil.isNotBlank(
                  scoreDTO.getUrl())));

      String fieldValue = StrUtil.nullToEmpty(scoreDTO.getName());
      // 根据是否有单位，取单位名称。没有为""
      String unit =
          fieldValue.contains("-") ? fieldValue.substring(fieldValue.lastIndexOf("-") + 1).trim()
              : "";
      // 如果有单位，先去掉单位
      if (StrUtil.isNotBlank(unit)) {
        int lastUnitPos = fieldValue.lastIndexOf("-") + 1;
        fieldValue = fieldValue.substring(0, lastUnitPos).replace("-", "").trim();
      }

      Tuple rankScore = convertRankScoreVal(fieldValue, rankItem.getUnitConvert());

      BigDecimal score = rankScore.get(0);
      String scoreStr;
      if (score == null) {
        scoreStr = fieldValue;
      } else {
        scoreStr = Convert.toStr(rankScore.get(1), "") + Convert.toStr(rankScore.get(2), "") + unit;
      }

      rankScoreBuilder.scoreStr(scoreStr).score(score).rankValue(1);

      RankScore rankScoreEntity = rankScoreBuilder.build();
      // 如果是第一个排名列，则设置默认排名列
      if (first) {
        rankData.setDefaultRankScoreStr(Convert.toStr(rankScoreEntity.getScore(), ""));
        rankData.setDefaultRankValue(rankScoreEntity.getRankValue());
      }

      // 添加到排名列列表
      rankData.getRankScoreList().add(rankScoreEntity);

    }
  }

  /**
   * 转换排名列值
   *
   * @param fieldValue  排名列值
   * @param unitConvert 单位转换
   * @return 转换后的排名列值
   */
  private Tuple convertRankScoreVal(String fieldValue, String unitConvert) {

    UnitConvertUtil unitConvertUtil = new UnitConvertUtil(unitConvert);
    if (StrUtil.isBlank(fieldValue)) {
      return unitConvertUtil.empty(null);
    }

    // 数据转换

    // 百分比
    if (PATTERN_REGEX_PERCENT.matcher(fieldValue).matches()) {
      return unitConvertUtil.empty(
          new BigDecimal(fieldValue.replace("%", "")).divide(ONE_HUNDRED, 2, RoundingMode.HALF_UP));
    }
    // 时间
    else if (PATTERN_REGEX_TIME.matcher(fieldValue).matches()) {
      String[] split = fieldValue.split(":");
      // 分秒
      if (split.length == 2) {
        return unitConvertUtil.empty(
            new BigDecimal(Integer.parseInt(split[0]) * 60 + Integer.parseInt(split[1])));
      }
      // 时分秒
      else if (split.length == 3) {
        return unitConvertUtil.empty(new BigDecimal(
            Integer.parseInt(split[0]) * 3600 + Integer.parseInt(split[1]) * 60 + Integer.parseInt(
                split[2])));
      } else {
        XxlJobHelper.log("转换排名数据列时间值为数字失败，fieldValue:{}，异常：{}", fieldValue);
        return unitConvertUtil.empty(null);
      }
    }
    // 千位符数字
    else if (StrUtil.contains(fieldValue, ",")) {
      return unitConvertUtil.convert(
          new BigDecimal(CharMatcher.inRange('0', '9').retainFrom(fieldValue)));
      // 字符串“-”
    } else if ("-".equals(fieldValue)) {
      return unitConvertUtil.empty(null);
    } else if (!NumberUtil.isNumber(fieldValue)) {
      return unitConvertUtil.empty(null);
    }
    // 数字
    else {
      try {
        return unitConvertUtil.convert(new BigDecimal(fieldValue));
      } catch (Exception e) {
        XxlJobHelper.log("转换排名数据列为数字失败，fieldValue:{}，异常：{}", fieldValue,
            ExceptionUtil.stacktraceToString(e));
        return unitConvertUtil.convert(
            Convert.toBigDecimal(CharMatcher.inRange('0', '9').retainFrom(fieldValue), null));
      }
    }

  }

  /**
   * 将数据按别名分组，然后根据别名找到原名称，再根据原名称找到映射后的名称，最后按映射后的名称分组
   *
   * @param rankDataList 排名数据集合
   * @param mappingMap   数据映射map
   * @return 分组后的数据
   */
  private Map<String, List<RankData>> convertRankDataMap(List<RankData> rankDataList,
      Map<String, String> mappingMap) {
    return rankDataList.stream()
        // 先按别名名称（如果没有别名就按名称）分组
        .collect(Collectors.groupingBy(
            item -> new RankDataNameDTO(StrUtil.blankToDefault(item.getAlias(), item.getName()))))
        .entrySet().stream()
        // 根据分组后的别名，找到原名称（只能找到一个，否则有问题），然后用原名称取数据映射map找映射后的名称再分组
        .collect(Collectors.groupingBy(item -> buildKey(item, mappingMap),
            Collectors.mapping(Map.Entry::getValue, Collectors.toList()))).entrySet().stream()
        // 分组完要将集合中的集合合并
        .collect(Collectors.toMap(Map.Entry::getKey,
            item -> item.getValue().stream().flatMap(Collection::stream)
                .collect(Collectors.toList())));
  }

  private Object checkDuplicate(List<RankData> collect, RankData result, Field field) {

    // 将字段值去空，去重
    List<Object> duplicateValueList = collect.stream()
        .map(rankData -> ReflectUtil.getFieldValue(rankData, field)).filter(ObjectUtil::isNotEmpty)
        .distinct().collect(Collectors.toList());

    // 如果过滤字段的数据集合为空，设置为null
    if (CollUtil.isEmpty(duplicateValueList)) {
      return null;
    }

    // 如果过滤字段的数据集合只有一个值，直接返回
    if (duplicateValueList.size() == 1) {
      return duplicateValueList.get(0);
    }

    String rankId = result.getRanks();
    String fieldName = field.getName();

    String tagList = "tagList";
    if (tagList.equals(fieldName)) {
      return duplicateRemovalTagList(rankId, duplicateValueList, collect);
    }

    // 下拉筛选项字符串
    String drops = "drops";
    if (drops.equals(fieldName)) {
      return duplicateRemovalDrops(rankId, duplicateValueList, collect);
    }

    // 下拉筛选项取值
    String rankDropList = "rankDropList";
    if (rankDropList.equals(fieldName)) {
      return duplicateRemovalRankDrop(rankId, duplicateValueList, collect);
    }

    // 排名列取值
    String rankScoreList = "rankScoreList";
    if (rankScoreList.equals(fieldName)) {
      return duplicateRemovalRankScore(rankId, duplicateValueList, collect);
    }

    // 如果存在多个告警
    // String content = StrUtil.format("排名【{}】数据清洗时存在字段值交叉，请关注！字段：{}，数据：\n{}",
    //     rankId, fieldName, JSONUtil.toJsonStr(collect));
    // WranMsgUtils.send(WranEnum.RANK.getTitle(), content);
    // throw new RankException(content);
    // 如果存在多个值，取第一个
    return duplicateValueList.get(0);
  }

  /**
   * 多数据源去重合并
   *
   * @param rankDataList 数据集合
   * @param mappingMap   数据映射map
   * @return 去重后数据集合
   */
  private List<RankData> duplicateRemoval(List<RankData> rankDataList,
      Map<String, String> mappingMap) {

    // key: 排名项名称。value：排名项数据（多数据源）
    Map<String, List<RankData>> rankDataMap = convertRankDataMap(rankDataList, mappingMap);

    // 不要参与去重的字段
    List<String> excludeFields = Lists.newArrayList(Fields.id, Fields.ranks, Fields.name,
        Fields.fullPinyinName, Fields.simplePinyinName, Fields.createDate, Fields.updateDate,
        Fields.delFlag, Fields.showHide, Fields.defaultRankItemId, Fields.groupDate, Fields.tags);
    // 需要去重的字段（通过不需要去重的字段反推）
    List<Field> fieldList = Arrays.stream(ReflectUtil.getFields(RankData.class))
        .filter(item -> !excludeFields.contains(item.getName())).collect(Collectors.toList());

    return rankDataMap.entrySet().stream()
        .map(item -> duplicateRemovalRankData(item, excludeFields, fieldList))
        .filter(Objects::nonNull).collect(Collectors.toList());
  }

  private Object duplicateRemovalTagList(String rankId, Object duplicateValueList,
      List<RankData> collect) {

    // [[';']]
    List<List<String>> duplicateValueTagList = (List<List<String>>) duplicateValueList;

    if (CollUtil.isEmpty(duplicateValueTagList) || duplicateValueTagList.size() == 1) {
      return duplicateValueList;
    }

    // 简单去重
    List<List<String>> tagList = duplicateValueTagList.stream().distinct()
        .collect(Collectors.toList());

    int listSize = tagList.size();
    if (listSize == 1) {
      return tagList;
    }

    int maxSize = tagList.stream().map(Collection::size).max(Integer::compareTo).orElse(0);

    String[][] tagArray = new String[listSize][maxSize];
    // 外层遍历，按最大长度遍历
    for (int i = 0; i < listSize; i++) {

      String[] tagArrayItem = new String[maxSize];
      List<String> itemList = tagList.get(i);

      // 内层遍历，按listSize遍历
      for (int j = 0; j < maxSize; j++) {

        if (j < itemList.size()) {
          tagArrayItem[j] = itemList.get(j);
        } else {
          tagArrayItem[j] = "";
        }

      }
      tagArray[i] = tagArrayItem;

    }

    List<String> resultList = Lists.newArrayList();
    for (int i = 0; i < maxSize; i++) {

      String[] temp = new String[listSize];
      for (int j = 0; j < listSize; j++) {
        temp[j] = tagArray[j][i];
      }

      List<String> itemTagList = Arrays.stream(temp).filter(StrUtil::isNotBlank).distinct()
          .collect(Collectors.toList());
      if (itemTagList.size() == 1) {
        resultList.add(itemTagList.get(0));
      } else if (itemTagList.size() > 1) {
        resultList.add(itemTagList.get(0));
        // String content = StrUtil.format(
        //     "排名【{}】数据清洗时存在下拉筛选项tags值交叉，请关注！数据：\n{}；\n实体数据：\n{}", rankId,
        //     JSONUtil.toJsonStr(duplicateValueTagList), JSONUtil.toJsonStr(collect));
        // WranMsgUtils.send(WranEnum.RANK.getTitle(), content);
      }

    }

    return resultList;
  }

  private Object duplicateRemovalDrops(String rankId, Object duplicateValueList,
      List<RankData> collect) {

    List<String> duplicateValueDropsList = (List<String>) duplicateValueList;

    if (CollUtil.isEmpty(duplicateValueDropsList) || duplicateValueDropsList.size() == 1) {
      return duplicateValueList;
    }

    List<String> dropsList = duplicateValueDropsList.stream().filter(StrUtil::isNotBlank)
        .filter(item -> ObjectUtil.notEqual(item, DROP_LIST)).distinct()
        .collect(Collectors.toList());

    if (CollUtil.isEmpty(dropsList)) {
      return DROP_LIST;
    }

    if (dropsList.size() == 1) {
      return dropsList.get(0);
    }

    List<JSONArray> dropsJsonArrayList = dropsList.stream().map(JSONArray::new)
        .collect(Collectors.toList());
    Integer size = dropsJsonArrayList.stream().map(List::size).findFirst().orElse(0);
    String[][] convert = new String[size][dropsList.size()];
    for (int i = 0; i < dropsJsonArrayList.size(); i++) {
      for (int j = 0; j < dropsJsonArrayList.get(i).size(); j++) {
        convert[j][i] = dropsJsonArrayList.get(i).getStr(j);
      }
    }

    List<String> result = Lists.newArrayList();
    for (String[] drop : convert) {
      List<String> drops = Arrays.stream(drop).filter(Objects::nonNull).filter(StrUtil::isNotBlank)
          .distinct().collect(Collectors.toList());
      if (CollUtil.isEmpty(drops)) {
        result.add(drop[0]);
        continue;
      }
      if (drops.size() == 1) {
        result.add(drops.get(0));
        continue;
      }
      // String content = StrUtil.format(
      //     "排名【{}】数据清洗时存在下拉筛选项drops值交叉，请关注！数据：\n{}；\n实体数据：\n{}", rankId,
      //     JSONUtil.toJsonStr(duplicateValueDropsList), JSONUtil.toJsonStr(collect));
      // WranMsgUtils.send(WranEnum.RANK.getTitle(), content);
      // throw new RankException(content);
      // 告警后，取第一个
      result.add(drops.get(0));
    }

    return JSONUtil.toJsonStr(
        result.stream().map(StrUtil::nullToEmpty).collect(Collectors.toList()));
  }

  private Object duplicateRemovalRankDrop(String rankId, Object duplicateValueList,
      List<RankData> collect) {
    List<List<RankDrop>> duplicateValueRankList = (List<List<RankDrop>>) duplicateValueList;

    if (CollUtil.isEmpty(duplicateValueRankList) || duplicateValueRankList.size() == 1) {
      return duplicateValueList;
    }

    if (duplicateValueRankList.stream().map(List::size).distinct().count() > 1) {
      String content = StrUtil.format(
          "排名【{}】数据清洗时存在下拉筛选项rankDrop数据长度不一致，请关注！，数据：\n{}；\n实体数据：\n{}",
          rankId, JSONUtil.toJsonStr(duplicateValueRankList), JSONUtil.toJsonStr(collect));
      // WranMsgUtils.send(WranEnum.RANK.getTitle(), content);
      throw new RankException(content);
    }

    Integer size = duplicateValueRankList.stream().map(List::size).findFirst().orElse(0);
    RankDrop[][] convert = new RankDrop[size][duplicateValueRankList.size()];
    for (int i = 0; i < duplicateValueRankList.size(); i++) {
      for (int j = 0; j < duplicateValueRankList.get(i).size(); j++) {
        convert[j][i] = duplicateValueRankList.get(i).get(j);
      }
    }

    List<RankDrop> result = Lists.newArrayList();
    for (RankDrop[] rankDrops : convert) {
      List<RankDrop> drops = Arrays.stream(rankDrops).filter(Objects::nonNull)
          .filter(item -> StrUtil.isNotBlank(item.getName())).distinct()
          .collect(Collectors.toList());
      if (CollUtil.isEmpty(drops)) {
        result.add(rankDrops[0]);
        continue;
      }
      if (drops.size() == 1) {
        result.add(drops.get(0));
        continue;
      }
      // String content = StrUtil.format(
      //     "排名【{}】数据清洗时存在下拉筛选项rankDrop值交叉，请关注！数据：\n{}；\n实体数据：\n{}", rankId,
      //     JSONUtil.toJsonStr(duplicateValueRankList), JSONUtil.toJsonStr(collect));
      // WranMsgUtils.send(WranEnum.RANK.getTitle(), content);
      // throw new RankException(content);
      // 告警后，取第一个
      result.add(drops.get(0));
    }

    return result;
  }

  private Object duplicateRemovalRankScore(String rankId, Object duplicateValueList,
      List<RankData> collect) {
    List<List<RankScore>> duplicateValueRankList = (List<List<RankScore>>) duplicateValueList;

    if (CollUtil.isEmpty(duplicateValueRankList) || duplicateValueRankList.size() == 1) {
      return duplicateValueList;
    }

    if (duplicateValueRankList.stream().map(List::size).distinct().count() > 1) {
      String content = StrUtil.format(
          "排名【{}】数据清洗时存在排名列rankScore数据长度不一致，请关注！，数据：\n{}；\n实体数据：\n{}",
          rankId, JSONUtil.toJsonStr(duplicateValueRankList), JSONUtil.toJsonStr(collect));
      // WranMsgUtils.send(WranEnum.RANK.getTitle(), content);
      throw new RankException(content);
    }

    Integer size = duplicateValueRankList.stream().map(List::size).findFirst().orElse(0);
    RankScore[][] convert = new RankScore[size][duplicateValueRankList.size()];
    for (int i = 0; i < duplicateValueRankList.size(); i++) {
      for (int j = 0; j < duplicateValueRankList.get(i).size(); j++) {
        convert[j][i] = duplicateValueRankList.get(i).get(j);
      }
    }

    List<RankScore> result = Lists.newArrayList();
    for (RankScore[] rankScores : convert) {
      List<RankScore> drops = Arrays.stream(rankScores).filter(Objects::nonNull)
          .filter(RankScore::getExist).filter(item -> StrUtil.isNotBlank(item.getScoreStr()))
          .distinct().collect(Collectors.toList());
      if (CollUtil.isEmpty(drops)) {
        continue;
      }
      if (drops.size() == 1) {
        result.add(drops.get(0));
        continue;
      }
      // String content = StrUtil.format(
      //     "排名【{}】数据清洗时存在排名列rankScore值交叉，请关注！数据：\n{}。\n转换后数据：\n{}；\n实体数据：\n{}",
      //     rankId, JSONUtil.toJsonStr(duplicateValueRankList), JSONUtil.toJsonStr(convert),
      //     JSONUtil.toJsonStr(collect));
      // WranMsgUtils.send(WranEnum.RANK.getTitle(), content);
      // throw new RankException(content);
      // 告警后，取第一个
      result.add(drops.get(0));
    }

    return result;
  }

  private RankData duplicateRemovalRankData(Map.Entry<String, List<RankData>> item,
      List<String> excludeFields, List<Field> fieldList) {

    // 先根据实体去重，将数据较新的覆盖掉较旧的数据
    List<RankData> collect = item.getValue().stream()
        // 先直接通过实体去重
        .distinct()
        /*// 时间分组
        .collect(Collectors.groupingBy(RankData::getGroupDate)).entrySet().stream()
        // 按时间倒序（最新数据在前）
        .sorted(Collections.reverseOrder(Comparator.comparingLong(
            entry -> entry.getKey().toInstant(ZoneOffset.of("+8")).toEpochMilli())))
        // 取第一条数据
        .limit(1).map(Map.Entry::getValue).flatMap(Collection::stream)*/.collect(
            Collectors.toList());

    // 如果去重后为空，直接返回null
    if (CollUtil.isEmpty(collect)) {
      return null;
    }

    String name = item.getKey();
    RankData data = collect.get(0);
    data.setName(name);
    data.setFullPinyinName(PinyinUtil.getPinyin(name, ""));
    data.setSimplePinyinName(PinyinUtil.getFirstLetter(name, ""));
    // 如果去重后只有一个，直接返回
    if (collect.size() == 1) {
      data.setTags(buildTags(data.getTagList()));
      return data;
    }

    // 根据字段取数据
    RankData result = new RankData();
    // 基本字段，取第一个实体的
    for (String excludeField : excludeFields) {
      if ("serialVersionUID".equals(excludeField)) {
        continue;
      }
      ReflectUtil.setFieldValue(result, excludeField,
          ReflectUtil.getFieldValue(data, excludeField));
    }

    // 数值字段，所有检查去重合并，如果存在同字段不同值任务会失败并告警
    for (Field field : fieldList) {
      String fieldName = field.getName();
      if ("serialVersionUID".equals(fieldName)) {
        continue;
      }
      ReflectUtil.setFieldValue(result, field, checkDuplicate(collect, result, field));
    }

    result.setTags(buildTags(result.getTagList()));
    return result;
  }

  /**
   * 本次排名与db排名合并
   *
   * @param rankDataDbEntityMap db排名数据
   * @param now                 本次数据源更新时间
   * @param item                本次排名数据
   */
  private void merge(Map<String, RankData> rankDataDbEntityMap, LocalDateTime now, RankData item) {
    // 处理本次数据源未更新的排名列
    List<RankScore> rankScoreList = item.getRankScoreList();
    if (CollUtil.isNotEmpty(rankScoreList)) {
      item.setRankScoreList(
          rankScoreList.stream().filter(RankScore::getExist).collect(Collectors.toList()));
    }
    RankData rankData = rankDataDbEntityMap.get(item.getId());
    if (rankData == null) {
      return;
    }
    item.setName(rankData.getName());
    item.setFullPinyinName(PinyinUtil.getPinyin(rankData.getName(), ""));
    item.setSimplePinyinName(PinyinUtil.getFirstLetter(rankData.getName(), ""));
    item.setImage(StrUtil.blankToDefault(item.getImage(), rankData.getImage()));
    item.setRecommendation(
        StrUtil.blankToDefault(item.getRecommendation(), rankData.getRecommendation()));
    // 别名 要么有值，要么为null，不能为空字符串
    String alias = StrUtil.blankToDefault(item.getAlias(), rankData.getAlias());
    item.setAlias(StrUtil.blankToDefault(alias, null));
    // 合并下拉筛选项
    mergeDrops(item, rankData);
    // 合并标签
    mergeTags(item, rankData);
    item.setUpdateDate(now);
  }

  /**
   * 将计算的排名数据与原始排名数据合并
   * <p></p>
   * 如果计算的排名数据中的数据为空，则使用原始排名数据中的数据，否则使用计算的排名数据中的数据
   *
   * @param item     计算的排名数据
   * @param rankData 原始排名数据
   */
  private void mergeDrops(RankData item, RankData rankData) {

    String itemDrops = StrUtil.blankToDefault(item.getDrops(), EMPTY_JSON);
    String rankDataDrops = StrUtil.blankToDefault(rankData.getDrops(), EMPTY_JSON);
    // 如果两个相同，直接返回
    if (StrUtil.equals(itemDrops, rankDataDrops)) {
      return;
    }

    // 默认值：["","",""]

    // 转换成list
    List<String> itemDropsList = JSONUtil.toList(JSONUtil.parseArray(itemDrops), String.class);
    List<String> rankDataDropsList = JSONUtil.toList(JSONUtil.parseArray(rankDataDrops),
        String.class);

    // 固定按照itemDropsList的长度来循环，因为itemDropsList的长度是固定的
    for (int i = 0; i < itemDropsList.size(); i++) {
      String itemDrop = itemDropsList.get(i);
      String rankDataDrop = rankDataDropsList.get(i);
      // 如果itemDrop为空，rankDataDrop不为空，就用rankDataDrop替换itemDrop
      if (StrUtil.isBlank(itemDrop) && StrUtil.isNotBlank(rankDataDrop)) {
        itemDropsList.set(i, rankDataDrop);
      }
    }

    // 清空item的dropList
    CollUtil.clear(item.getRankDropList());
    // 构建新的dropList
    buildRankDropList(item, item.getRanks());

  }

  /**
   * 将计算的排名数据标签与原始排名数据标签合并
   * <p></p>
   * 优先级：计算的排名数据标签 > 原始排名数据标签
   *
   * @param item     计算的排名数据
   * @param rankData 原始排名数据
   */
  private void mergeTags(RankData item, RankData rankData) {

    // 转换成JSONObject
    JSONObject itemTags = JSONUtil.parseObj(StrUtil.blankToDefault(item.getTags(), "{}"), false);
    JSONObject rankDataTags = JSONUtil.parseObj(StrUtil.blankToDefault(rankData.getTags(), "{}"),
        false);

    // 前置判断
    if (itemTags.isEmpty() && rankDataTags.isEmpty()) {
      return;
    }
    if (itemTags.equals(rankDataTags)) {
      return;
    }
    if (rankDataTags.isEmpty()) {
      return;
    }
    if (itemTags.isEmpty()) {
      item.setTags(rankData.getTags());
      return;
    }

    Iterator<Entry<String, Object>> itemTagsIterator = itemTags.iterator();
    Iterator<Entry<String, Object>> rankDataTagsIterator = rankDataTags.iterator();

    JSONObject result = JSONUtil.createObj(JSONConfig.create().setIgnoreNullValue(false));
    // 以最大的长度为循环次数
    for (int i = 0; i < Math.max(itemTags.size(), rankDataTags.size()); i++) {

      Entry<String, Object> itemEntry = itemTagsIterator.hasNext() ? itemTagsIterator.next() : null;
      Entry<String, Object> rankDataEntry =
          rankDataTagsIterator.hasNext() ? rankDataTagsIterator.next() : null;

      // 如果两个都为空，直接跳过，并设置数据生成中标签（占位）
      if (itemEntry == null && rankDataEntry == null) {
        result.set("其他" + (i + 1), "数据生成中");
        continue;
      }

      // 如果itemEntry为空，rankDataEntry不为空，直接设置rankDataEntry
      if (itemEntry == null) {
        result.set(rankDataEntry.getKey(), rankDataEntry.getValue());
        continue;
      }

      // 如果rankDataEntry为空，itemEntry不为空，直接设置itemEntry
      if (rankDataEntry == null) {
        result.set(itemEntry.getKey(), itemEntry.getValue());
        continue;
      }

      // 如果两个的key相同
      if (StrUtil.equals(itemEntry.getKey(), rankDataEntry.getKey())) {
        // 如果itemEntry的值为空，直接设置rankDataEntry的值
        if (StrUtil.isBlankIfStr(itemEntry.getValue())) {
          result.set(itemEntry.getKey(), rankDataEntry.getValue());
        }
        // 否则直接设置itemEntry的值
        else {
          result.set(itemEntry.getKey(), itemEntry.getValue());
        }
        continue;
      }

      // 如果两个的key不相同，且itemEntry的key以“其他”开头，直接设置rankDataEntry的值
      if (StrUtil.startWith(itemEntry.getKey(), "其他")) {
        result.set(rankDataEntry.getKey(), rankDataEntry.getValue());
        continue;
      }

      // 如果两个的key不相同，且rankDataEntry的key以“其他”开头，直接设置itemEntry的值
      if (StrUtil.startWith(rankDataEntry.getKey(), "其他")) {
        result.set(itemEntry.getKey(), itemEntry.getValue());
        continue;
      }

      // 如果两个的key不相同，且itemEntry的key不以“其他”开头，直接设置itemEntry的值，并告警
      result.set(itemEntry.getKey(), itemEntry.getValue());
      // String content = StrUtil.format(
      //     "合并tags时，两个key不相同，且itemEntry的key不以“其他”开头，直接设置itemEntry的值，itemEntry的key：{}， rankDataEntry的key：{}",
      //     itemEntry.getKey(), rankDataEntry.getKey());
      // WranMsgUtils.send(WranEnum.RANK.getTitle(), content);

    }

    item.setTags(result.toString());
  }

  /**
   * 通过本次排名数据的名称，查询db中排名下所有的排名数据
   *
   * @param rankId             排名id
   * @param rankDataList       本次排名数据
   * @param rankDataMappingMap 本次排名数据的名称映射
   * @return db中排名下所有的排名数据
   */
  private List<RankData> getDbRankDataList(String rankId, List<RankData> rankDataList,
      Map<String, String> rankDataMappingMap) {

    // 本次排名数据的名称
    List<String> rankDataNameList = rankDataList.stream().map(RankData::getName)
        .collect(Collectors.toList());
    // 本次排名数据的名称映射
    rankDataNameList.stream().map(rankDataMappingMap::get).filter(Objects::nonNull)
        .forEach(rankDataNameList::add);

    // 查询db中排名下所有的排名数据
    List<RankData> dbRankDataList = rankDataService.list(
        Wrappers.<RankData>lambdaQuery().eq(RankData::getRanks, rankId)
            .in(RankData::getName, rankDataNameList));
    // 通过别名查询db中排名下所有的排名数据
    List<RankData> dbAliasRankDataList = rankDataService.list(
        Wrappers.<RankData>lambdaQuery().eq(RankData::getRanks, rankId)
            .in(RankData::getAlias, rankDataNameList));

    dbRankDataList.addAll(dbAliasRankDataList);

    // 去重
    dbRankDataList = dbRankDataList.stream().collect(Collectors.collectingAndThen(
        Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(RankData::getName))),
        CollUtil::newArrayList));
    return dbRankDataList;
  }

  /**
   * 本次排名数据，与db中的排名数据进行匹配，匹配到的设置id
   *
   * @param rankDataList         本次排名数据
   * @param rankDataDbMap        db中排名数据的名称映射
   * @param rankDataAliasMap     db中排名数据的别名映射
   * @param rankDataMappingBiMap 本次排名数据的名称映射
   */
  private void matchRankDataId(List<RankData> rankDataList, Map<String, String> rankDataDbMap,
      Map<String, String> rankDataAliasMap, Map<String, List<String>> rankDataMappingBiMap) {

    if (CollUtil.isEmpty(rankDataList)) {
      return;
    }

    rankDataList.forEach(item -> {

      // 名称
      String itemName = item.getName();

      // 通过名称匹配
      String id = rankDataDbMap.get(itemName);

      if (id != null) {
        // 如果匹配到了，设置id并结束
        item.setId(id);
        return;
      }

      // 通过别名匹配
      id = rankDataAliasMap.get(itemName);

      if (id != null) {
        // 如果匹配到了，设置id并结束
        item.setId(id);
        return;
      }

      List<String> mappingNameList = rankDataMappingBiMap.get(itemName);
      if (CollUtil.isEmpty(mappingNameList)) {
        return;
      }

      // 通过映射关系，获取映射前的名称
      for (String mappingName : mappingNameList) {
        // 通过映射前的名称匹配
        id = rankDataDbMap.get(mappingName);
        if (id != null) {
          // 如果匹配到了，设置id并结束
          item.setId(id);
          break;
        }
      }

    });
  }

  /**
   * 检查别名是否重复
   */
  private void checkAliasRepeat(String rankId, List<RankData> addList, List<RankData> updateList) {

    if (CollUtil.isEmpty(addList) && CollUtil.isEmpty(updateList)) {
      return;
    }

    // 检查新增的排名数据的别名是否重复
    checkAddListAliasRepeat(rankId, addList);

    // 检查更新的排名数据的别名是否重复
    checkUpdListAliasRepeat(rankId, updateList);

  }

  /**
   * 检查新增的排名数据的别名是否重复
   *
   * @param addList 新增的排名数据
   * @throws RankException 重复别名
   */
  private void checkAddListAliasRepeat(String rankId, List<RankData> addList) {
    // 检查新增的排名数据的别名是否重复
    if (CollUtil.isEmpty(addList)) {
      return;
    }

    List<String> addAliasList = addList.stream().map(RankData::getAlias).filter(Objects::nonNull)
        .collect(Collectors.toList());

    if (CollUtil.isEmpty(addAliasList)) {
      return;
    }

    List<RankData> addAliasRepeatList = rankDataService.list(
        Wrappers.<RankData>lambdaQuery().eq(RankData::getRanks, rankId)
            .in(RankData::getAlias, addAliasList)
            .select(RankData::getId, RankData::getName, RankData::getAlias));

    if (CollUtil.isNotEmpty(addAliasRepeatList)) {
      throw new RankAliasRepeatException(
          StrUtil.format("新增的排名数据的别名重复。原数据：{}，新数据：{}", addAliasRepeatList,
              addList.stream().filter(item -> addAliasList.contains(item.getAlias()))
                  .collect(Collectors.toList())));
    }

  }

  /**
   * 检查更新的排名数据的别名是否重复
   *
   * @param updateList 更新的排名数据
   * @throws RankException 重复别名
   */
  private void checkUpdListAliasRepeat(String rankId, List<RankData> updateList) {

    if (CollUtil.isEmpty(updateList)) {
      return;
    }

    List<String> updateAliasList = updateList.stream().map(RankData::getAlias)
        .filter(Objects::nonNull).collect(Collectors.toList());

    if (CollUtil.isEmpty(updateAliasList)) {
      return;
    }

    List<String> idList = updateList.stream().map(RankData::getId).collect(Collectors.toList());

    List<String> updateAliasRepeatList = rankDataService.listObjs(
        Wrappers.<RankData>lambdaQuery().eq(RankData::getRanks, rankId)
            .in(RankData::getAlias, updateAliasList).notIn(RankData::getId, idList)
            .select(RankData::getId), Convert::toStr);

    // 取交集
    Collection<String> intersection = CollUtil.intersection(updateAliasList, updateAliasRepeatList);

    // 如果交集不为空，说明有重复的别名
    if (CollUtil.isNotEmpty(intersection)) {
      throw new RankAliasRepeatException(
          "更新的排名数据的别名重复。原数据：" + updateAliasRepeatList + "，新数据："
              + updateList.stream().filter(item -> intersection.contains(item.getAlias()))
              .collect(Collectors.toList()) + "");
    }

  }

}
